# -*- coding: utf-8 -*-
# @Time    : 2024/1/25 20:24
# @Author  : micah
# @File    : 6.使用异步的方式获取汽车之家数据.py
# @Software: PyCharm


import redis
import hashlib
import asyncio
import aiohttp
import aiomysql
import chardet
from lxml import etree


class CarSpider:
    redis_client = redis.Redis()

    def __init__(self):
        self.url = 'https://www.che168.com/china/a0_0msdgscncgpi1ltocsp{}exf4x0/?pvareaid=102179#currengpostion'
        self.api_url = 'https://cacheapigo.che168.com/CarProduct/GetParam.ashx?specid={}'

        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
        }

    def __del__(self):
        self.redis_client.close()

    # 获取汽车id
    async def get_car_id(self, page, session, pool):
        """
        :param page: 翻页参数
        :param session: 请求对象
        :param pool: 数据库连接池对象
        :return:
        """
        async with session.get(self.url.format(page), headers=self.headers) as response:
            # 使用response对象获取原始数据
            content = await response.read()

            # 获取返回的页面的编码集
            # 如果返回的页面编码集为utf-8则页面显示的内容是访问过快
            encoding = chardet.detect(content)['encoding']

            if encoding == 'GB2312' or encoding == 'ISO-8859-1':
                result = content.decode('gbk')
            else:
                result = content.decode(encoding)
                print(f'访问频繁: {encoding}')

            tree = etree.HTML(result)
            id_list = tree.xpath('//ul[@class="viewlist_ul"]/li/@specid')
            if id_list:
                tasks = [asyncio.create_task(self.get_car_info(spec_id, session, pool)) for spec_id in id_list]
                await asyncio.wait(tasks)

    async def get_car_info(self, spec_id, session, pool):
        async with session.get(self.api_url.format(spec_id), headers=self.headers) as response:
            json_data = await response.json()
            if json_data['result'].get('paramtypeitems'):
                item = dict()
                item['name'] = json_data['result']['paramtypeitems'][0]['paramitems'][0]['value']
                item['price'] = json_data['result']['paramtypeitems'][0]['paramitems'][1]['value']
                item['brand'] = json_data['result']['paramtypeitems'][0]['paramitems'][2]['value']
                item['altitude'] = json_data['result']['paramtypeitems'][1]['paramitems'][2]['value']
                item['breadth'] = json_data['result']['paramtypeitems'][1]['paramitems'][1]['value']
                item['length'] = json_data['result']['paramtypeitems'][1]['paramitems'][0]['value']

                # 当数据提取完成之后要进行保存
                await self.save_car_info(item, pool)
            else:
                print('数据不存在...')

    # 在数据保存之前需要将数据编码成md5并完成去重
    @staticmethod
    def get_md5(dict_item):
        md5 = hashlib.md5()
        md5.update(str(dict_item).encode('utf-8'))
        return md5.hexdigest()

    async def save_car_info(self, item, pool):
        # 使用异步上下文管理器的方式完成数据保存
        async with pool.acquire() as conn:
            async with conn.cursor() as cursor:
                val_md5 = self.get_md5(item)
                redis_result = self.redis_client.sadd('car:filter', val_md5)
                if redis_result:
                    sql = """
                        insert into car_info() values (
                            %s, %s, %s, %s, %s, %s, %s
                        );
                    """
                    try:
                        await cursor.execute(sql, (
                            0,
                            item['name'],
                            item['price'],
                            item['brand'],
                            item['altitude'],
                            item['breadth'],
                            item['length']))

                        await conn.commit()
                        print('插入成功:', item)
                    except Exception as e:
                        print('插入失败:', e)
                        await conn.rollback()

    async def main(self):
        # 创建数据库连接池
        async with aiomysql.create_pool(user='root', password='123456', host='localhost', port=3306, db='py_spider') as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cursor:
                    # 创建表
                    create_table_sql = """
                        create table car_info(
                            id int primary key auto_increment,
                            name varchar(100),
                            price varchar(100),
                            brand varchar(100),
                            altitude varchar(100),
                            breadth varchar(100),
                            length varchar(100)
                        );
                    """

                    # 在异步代码中必须先要检查表是否存在, 直接使用if not语句无效
                    # 在执行后返回一个结果: 1[存在] 0[不存在]
                    check_table_query = "show tables like 'car_info'"
                    result = await cursor.execute(check_table_query)

                    if not result:
                        await cursor.execute(create_table_sql)
                        print('表创建成功...')
                    else:
                        print('表已存在...')

            # 创建请求对象
            async with aiohttp.ClientSession() as session:
                tasks = [asyncio.create_task(self.get_car_id(page, session, pool)) for page in range(1, 11)]
                await asyncio.wait(tasks)


if __name__ == '__main__':
    car_spider = CarSpider()
    asyncio.run(car_spider.main())
